{"componentChunkName":"component---src-templates-bootcamp-js","path":"/bootcamp/bigdata/1-bigdata/","result":{"data":{"site":{"siteMetadata":{"title":"William Blogs & More","description":"team knowledge base"}},"allBootcamp":{"edges":[{"node":{"fields":{"id":"410a853e-19db-5ecf-9193-d90fd4e2b164","slug":"/README/","title":"README"}}},{"node":{"fields":{"id":"ebd8a7a9-4f93-5f7f-89b4-7a921fc850fd","slug":"/backend/DEV/","title":"DEV"}}},{"node":{"fields":{"id":"c97bf13a-dd3f-5e0b-9f0a-9c5f08a42d57","slug":"/backend/Restful/","title":"Restful"}}},{"node":{"fields":{"id":"81e9e69e-bf6a-51e5-843d-7b4321c330f9","slug":"/backend/curl/","title":"Curl"}}},{"node":{"fields":{"id":"2aa2ab91-e4c3-59d3-ae10-2949c78bbba2","slug":"/backend/express/","title":"Express"}}},{"node":{"fields":{"id":"923aee4e-1ea6-5b5c-9dbf-7f3ed2ccd8de","slug":"/backend/flow/","title":"Flow"}}},{"node":{"fields":{"id":"735e010f-fe92-58c2-a717-44e1379ec81a","slug":"/backend/microservice/","title":"Microservice"}}},{"node":{"fields":{"id":"5257ef00-003a-505c-9681-90b156b2fdc3","slug":"/backend/mongo/","title":"Mongo"}}},{"node":{"fields":{"id":"5d7e4d14-bbe0-51e7-ac00-aed822be865d","slug":"/backend/nodejs/","title":"Nodejs"}}},{"node":{"fields":{"id":"04e0f6cd-ba6d-5bcb-a711-f978053c85a9","slug":"/backend/pg/","title":"Pg"}}},{"node":{"fields":{"id":"07933914-1441-5d9a-93b3-7b8527f32834","slug":"/backend/rabbitmq/","title":"Rabbitmq"}}},{"node":{"fields":{"id":"aeee00c3-e65a-5e57-8e26-99bdee4b5972","slug":"/backend/redis/","title":"Redis"}}},{"node":{"fields":{"id":"5efa4932-4a43-5627-a52e-d6c44ea1ce37","slug":"/backend/socket/","title":"Socket"}}},{"node":{"fields":{"id":"2546451c-541b-5d1f-b34c-1f1842685cfc","slug":"/bestitconsulting/Bootcamp/1-bootcamp/","title":"1 Bootcamp"}}},{"node":{"fields":{"id":"3ce1ed0e-ce4c-53b6-89a3-dd5dc7642647","slug":"/bestitconsulting/Bootcamp/2-release_branch/","title":"2 Release_branch"}}},{"node":{"fields":{"id":"e3f26574-c4e6-5058-8336-6e761d369455","slug":"/bestitconsulting/Bootcamp/3-coding/","title":"3 Coding"}}},{"node":{"fields":{"id":"8b752412-b3d3-5bbe-87b3-d99ef91d2759","slug":"/bestitconsulting/Bootcamp/4-testing/","title":"4 Testing"}}},{"node":{"fields":{"id":"e950ea75-d6c1-5fca-8e1c-4b4fce362fab","slug":"/bestitconsulting/Bootcamp/5-CICD/","title":"5 CICD"}}},{"node":{"fields":{"id":"739b0238-56b4-5a47-bc76-3286eadcb425","slug":"/bestitconsulting/Bootcamp/6-BA/","title":"6 BA"}}},{"node":{"fields":{"id":"bc2e6ad3-6b39-54b0-8d66-e5e295f331ee","slug":"/bestitconsulting/Bootcamp/QA/","title":"QA"}}},{"node":{"fields":{"id":"484882b7-848a-591f-80d5-24d045cb9582","slug":"/bestitconsulting/Bootcamp/README/","title":"README"}}},{"node":{"fields":{"id":"27252ae9-1261-5b83-9039-45c41697c09d","slug":"/bestitconsulting/Bootcamp/Reference/","title":"Reference"}}},{"node":{"fields":{"id":"2a271bc5-0775-5cb0-99ab-056d5c0cf09f","slug":"/bestitconsulting/courses/agile/","title":"Agile"}}},{"node":{"fields":{"id":"372866f9-ffd2-5576-bbc1-415732492686","slug":"/bestitconsulting/courses/grow-admin/","title":"Grow Admin"}}},{"node":{"fields":{"id":"5eb7bdc4-c171-52bc-bf48-f413de03ea24","slug":"/bestitconsulting/courses/learn-suite/","title":"Learn Suite"}}},{"node":{"fields":{"id":"986b6381-1b35-5253-93d5-1294d4984202","slug":"/bestitconsulting/courses/security/","title":"Security"}}},{"node":{"fields":{"id":"8e3b3436-48d9-57f8-9d53-71494461a76e","slug":"/bestitconsulting/radar/github-radar-1/","title":"Github Radar 1"}}},{"node":{"fields":{"id":"f9b1b128-a118-5569-b2ee-b6d5c347e964","slug":"/bestitconsulting/radar/github-radar/","title":"Github Radar"}}},{"node":{"fields":{"id":"17a02ec4-b391-54f1-8c60-922492f1a5eb","slug":"/bigdata/1-bigdata/","title":"1 Bigdata"}}},{"node":{"fields":{"id":"36884170-cd04-548a-9a00-77e36a4b1380","slug":"/bigdata/2-hadoop/","title":"2 Hadoop"}}},{"node":{"fields":{"id":"c9cdff49-84d1-58e6-bf25-926b7f68d84c","slug":"/bigdata/201-kafka-4/","title":"201 Kafka 4"}}},{"node":{"fields":{"id":"f2387a56-a470-54c8-94aa-182fffbab8b0","slug":"/bigdata/201-nosql-8/","title":"201 Nosql 8"}}},{"node":{"fields":{"id":"4d3d0dd0-c311-51f6-9975-80fd4c5fa752","slug":"/bigdata/201-streaming-5/","title":"201 Streaming 5"}}},{"node":{"fields":{"id":"27a22e9e-6aa2-5c4e-9edf-c06b8740246d","slug":"/bigdata/3-hdfs/","title":"3 Hdfs"}}},{"node":{"fields":{"id":"f80ad3db-5700-5e3b-a5ae-2e6f5dc0c886","slug":"/bigdata/4-devops/","title":"4 Devops"}}},{"node":{"fields":{"id":"07672f34-d70e-589c-ba8e-67e284bc1d8c","slug":"/bigdata/5-hive/","title":"5 Hive"}}},{"node":{"fields":{"id":"bf3a9766-31fd-5213-8dd9-7c38ebe7bb80","slug":"/bigdata/6-spark/","title":"6 Spark"}}},{"node":{"fields":{"id":"4dab467f-b6ea-531b-8c28-9bcef71863e4","slug":"/bigdata/7-docker/","title":"7 Docker"}}},{"node":{"fields":{"id":"7ee4ff75-d3f8-5c1a-b701-facf126f3450","slug":"/bigdata/8-streaming/","title":"8 Streaming"}}},{"node":{"fields":{"id":"6785ce66-99e5-5e00-af30-b1c32c61cb12","slug":"/bigdata/9-elasticsearch/","title":"9 Elasticsearch"}}},{"node":{"fields":{"id":"9a59f47c-4cd3-51db-9962-2ffb26b14dc4","slug":"/bigdata/hdfs-kubernetes/","title":"Hdfs Kubernetes"}}},{"node":{"fields":{"id":"819a354c-d042-54db-9466-5063d21db45f","slug":"/bigdata/kubernetes/","title":"Kubernetes"}}},{"node":{"fields":{"id":"4535d21c-0f79-5cda-b235-98658a1ffe1f","slug":"/bigdata/spark-kubernetes/","title":"Spark Kubernetes"}}},{"node":{"fields":{"id":"07d1d0ca-dc16-52b5-b911-0d128a2388eb","slug":"/bigdata2/README/","title":"README"}}},{"node":{"fields":{"id":"605faba4-a685-5a80-a80a-5130700509e3","slug":"/bigdata2/TODO/","title":"TODO"}}},{"node":{"fields":{"id":"5b1a9498-e94f-5dba-a4fa-9aa2f3da4312","slug":"/bigdata2/editor/","title":"Editor"}}},{"node":{"fields":{"id":"6e1973a3-4045-57b9-a096-d498f5d2802b","slug":"/bigdata2/hadoop/","title":"Hadoop"}}},{"node":{"fields":{"id":"e63f21ad-65e0-56db-891e-60a89c5f93da","slug":"/bigdata2/kafka/","title":"Kafka"}}},{"node":{"fields":{"id":"2efc1fee-93bd-53eb-b5e0-672ff2acc9a4","slug":"/bigdata2/scala/","title":"Scala"}}},{"node":{"fields":{"id":"001e1d27-4cba-5926-ba33-8197e2065136","slug":"/bigdata2/spark/","title":"Spark"}}},{"node":{"fields":{"id":"2e2078b0-b734-5ae3-873c-022349c1c3cf","slug":"/bigdata2/vscode/","title":"Vscode"}}},{"node":{"fields":{"id":"b52de8e7-4791-556a-9989-b84717cd3370","slug":"/bigdata2/web-resource/","title":"Web Resource"}}},{"node":{"fields":{"id":"e0378d10-6064-560e-a6b2-cbe89a820d26","slug":"/bigdata2/zookeeper/","title":"Zookeeper"}}},{"node":{"fields":{"id":"616110f5-9073-5438-846b-ff7584d4e922","slug":"/cloud/aws/","title":"Aws"}}},{"node":{"fields":{"id":"c788368a-0f0e-5661-8fe9-c469d5a278d8","slug":"/cloud/azure/","title":"Azure"}}},{"node":{"fields":{"id":"30bec85f-fc28-5638-b42d-a952b414de7c","slug":"/cloud/elk/","title":"Elk"}}},{"node":{"fields":{"id":"57773e69-6a69-5e86-a641-af37ffe1ad40","slug":"/cloud/gcp/","title":"Gcp"}}},{"node":{"fields":{"id":"6688fe10-a20c-5479-bc80-170063d3e58c","slug":"/frontend/apollo/","title":"Apollo"}}},{"node":{"fields":{"id":"09e2c05f-9d10-5776-8fe8-44e9a565a20e","slug":"/frontend/graphql/","title":"Graphql"}}},{"node":{"fields":{"id":"28d38d20-c6e2-50ae-ac83-d3e3b9f25248","slug":"/frontend/proxy/","title":"Proxy"}}},{"node":{"fields":{"id":"ca4444ea-9796-5d40-98e1-c01d80f88221","slug":"/frontend/react-router/","title":"React Router"}}},{"node":{"fields":{"id":"61e06420-e9e4-5aa4-9d71-bf618be3f4fe","slug":"/frontend/react/","title":"React"}}},{"node":{"fields":{"id":"9d95393b-96be-5065-83bb-536b746eed21","slug":"/frontend/redux/","title":"Redux"}}},{"node":{"fields":{"id":"e3bb0f44-4819-556c-8511-7402303e36c5","slug":"/misc/0428/","title":"0428"}}},{"node":{"fields":{"id":"dfcc8a86-17af-5c8c-b831-9fe4fa51bf0e","slug":"/misc/HISTORY/","title":"HISTORY"}}},{"node":{"fields":{"id":"6f004260-a55d-5a2d-8146-d197509fd980","slug":"/misc/TODO/","title":"TODO"}}},{"node":{"fields":{"id":"30096e9c-2593-5fc8-80d5-1f7648898888","slug":"/misc/git/","title":"Git"}}},{"node":{"fields":{"id":"bed21539-b0d7-5936-9c15-96905db68ac9","slug":"/misc/gitlab/","title":"Gitlab"}}},{"node":{"fields":{"id":"ab43544c-0ef1-56d2-a0a3-035fdd1cf7fe","slug":"/misc/installation/","title":"Installation"}}},{"node":{"fields":{"id":"b2e02e77-9623-5967-a541-fe1829315b47","slug":"/misc/misc/","title":"Misc"}}},{"node":{"fields":{"id":"4c7d96dc-fddc-52fd-9333-9fa6d2633874","slug":"/misc/vocabulary/","title":"Vocabulary"}}},{"node":{"fields":{"id":"f2a8cc1a-8b27-541c-b68e-8c5dc6d0f6b3","slug":"/poc/courses/agile/","title":"Agile"}}},{"node":{"fields":{"id":"ae6e68d8-0f75-5fac-ad96-f042bddd50aa","slug":"/poc/courses/grow-admin/","title":"Grow Admin"}}},{"node":{"fields":{"id":"689b1b72-e7b6-5b07-9890-50f653fdd7df","slug":"/poc/courses/learn-suite/","title":"Learn Suite"}}},{"node":{"fields":{"id":"003c5b81-20aa-57aa-b268-17aaaa338e68","slug":"/poc/courses/security/","title":"Security"}}},{"node":{"fields":{"id":"e0f53ac9-fbe0-5c72-8b69-ee2cce387f90","slug":"/poc/radar/github-radar-1/","title":"Github Radar 1"}}},{"node":{"fields":{"id":"c5718e38-5cc1-566e-a649-a4d501877995","slug":"/poc/radar/github-radar/","title":"Github Radar"}}},{"node":{"fields":{"id":"5a68cfb3-1b53-5527-a6ad-b8277381fab4","slug":"/poc/radar/steps/","title":"Steps"}}}]},"bootcamp":{"fields":{"id":"17a02ec4-b391-54f1-8c60-922492f1a5eb","title":"1 Bigdata","slug":"/bigdata/1-bigdata/","tag":"bigdata","category":"bootcamp","date":"2020-12-17","size":1953},"html":"<h3 id=\"5-v\" style=\"position:relative;\"><a href=\"#5-v\" aria-label=\"5 v permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>5 V</h3>\n<ul>\n<li>volume</li>\n<li>velocity</li>\n<li>variety</li>\n</ul>\n<h3 id=\"data-type\" style=\"position:relative;\"><a href=\"#data-type\" aria-label=\"data type permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>data type</h3>\n<ul>\n<li>structured data: csv, tsv, DB</li>\n<li>semi-structured data - 5-10%: logs, xml, json</li>\n<li>unstructured data: email, social media, sounds, vedio,images, word</li>\n</ul>\n<h3 id=\"speed-of-data\" style=\"position:relative;\"><a href=\"#speed-of-data\" aria-label=\"speed of data permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Speed of data</h3>\n<ul>\n<li>batch</li>\n<li>micro-batch</li>\n<li>real-time</li>\n</ul>\n<p>Speed of data = from real time to batched processing</p>\n<pre>\n    File Storage: HDFS, ZFS\n\n    No-SQL:\n    - `Key-Value Store`: Redis, Aerospike\n    - `In-memory`: Ignite\n    - `Distributed DB`: Cassandra, HBase\n\n    Tranditional DB: MySQL, Postgres\n\n    Raw File Formats\n    \n    Typical access Pattern:\n\n    - Use all the fields to validata, enhance, join data\n    - Read through whole data set\n\n    Formats:\n    - Plain Text (unstructured)\n    - Structured Text Data (Rows - CSV, TSV, JSON)\n    - Binary files (images, videos)\n    - Avro (Row oriented)\n   \n    Processed File Formats\n\n    Typical access Pattern:\n    - Use limited fields to aggregate data or run other analytical queries\n    - Read filtered subset\n\n    Formats:\n\n    - Parquet (Columnar oriented)\n    - ORC\n</pre> \n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/bbe82fa01bbfc8e04e90297c73fe8d4a/21335/bigdata1.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 51.33333333333333%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAAA7DAAAOwwHHb6hkAAABeklEQVQoz2VS2VLDMAzM//8d0MIT9MoFNHF9JHbOZeUmnbZoRrFjy7uSVknTNNBawzUOzjlYZ9G0DYwxGMcR0zRFn+f5wdfzZ0/OPqBwLXLn8es7BuNm92DDMGDoexjvkWsDK/uuhwodNH1cAX3bQtUVZJ2emOfp+g+ShBCQFxlO6QlplsJaC0/wlt4TfM060WS59AM6eShZxYvHsmQfuhCBquqMuq5RqzqSCGjP7G+AJdP/zHIc8gwXpRaQBWy+Aop5Pn55fcH7xxab7QaH4yH2XPHNQ4Y9mXMy5wSspHQyrv1bS16Dy+8S+8MOGeMFbKJoItwcY5ceaqr5tnll4D4y3YPN96BrplwDz8QdS/XDGAXql/skdB0yZrdnCUVZwFjzbyzEpPnSwx3jxJXRaHxL97AcPRk1qS4pOTKpcchtg5N2qDg6eFZ7EeWUHvHz8w11UZD5FVHkPAQfwSKgfC4MkOE2ZH1W9yYK4752X1FlRYVFjPl+aBf7AwFgCDJGBq3+AAAAAElFTkSuQmCC'); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Storage &amp; formats of data\"\n        title=\"Storage &amp; formats of data\"\n        src=\"/static/bbe82fa01bbfc8e04e90297c73fe8d4a/0a47e/bigdata1.png\"\n        srcset=\"/static/bbe82fa01bbfc8e04e90297c73fe8d4a/8a4e8/bigdata1.png 150w,\n/static/bbe82fa01bbfc8e04e90297c73fe8d4a/5a46d/bigdata1.png 300w,\n/static/bbe82fa01bbfc8e04e90297c73fe8d4a/0a47e/bigdata1.png 600w,\n/static/bbe82fa01bbfc8e04e90297c73fe8d4a/1cfc2/bigdata1.png 900w,\n/static/bbe82fa01bbfc8e04e90297c73fe8d4a/21335/bigdata1.png 1082w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h2 id=\"storage--formats-of-data\" style=\"position:relative;\"><a href=\"#storage--formats-of-data\" aria-label=\"storage  formats of data permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Storage &#x26; Formats of data</h2>\n<ul>\n<li>Fow Format - OOTP, query faster</li>\n<li>Columnar format - OLAP, historical/archive data</li>\n</ul>\n<h2 id=\"data-compression\" style=\"position:relative;\"><a href=\"#data-compression\" aria-label=\"data compression permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Data compression</h2>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/71ccccd89471c671316bf30da27bfd97/60b3a/datacompress.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 61.33333333333334%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAMCAYAAABiDJ37AAAACXBIWXMAAA7DAAAOwwHHb6hkAAACBUlEQVQoz11S2W4TQRDcr4YAL4EH4JmPIopASDh5IIkTWT5ibO85M3vNHkXVrA+JlVrbPd1dXVM90YfbR1zdPuEd7e3NA65oX3894zPtDX3Zp59zXP94OsdfmPt4jNWj2vfE+TZbIPq9PeD7fIGblyVmf2M8Zg73uySY/IfM4o7+3TGW/UkK/u05nm33uN9nWJgSUedbmCyByVOMnYc+W2QoTYHpG0OutOYcV86gqRxOnzM5fF2FXFSUJdKioOVwdQ0/DMiMgc7bfqD1SAluGCvXdD1ya5E7G3KqKUqHsmnQMI7arkNBgJxNwzii56EhG50NBOiHHsUpZl5Ws9mzbxhYLwI5b1RV6OhHtiqx2qyx3r6iatpw6Jis1MTpngMucf9f/hLXbRMYR0osV0tsXjdovcdIBp5aiqkYMAxsxJRRYNS27VRDX4zVp3o/HK+cUnRdWdM0QHomWYqGC7PU7pAmiGnyewJYalY1dQBUjXplNYEDYN3W1GUqqLgYNahgAnRYk71kkd+PZMjz9sgw5/B9vMfusIPl1aNOGjluidP1aSGr9QprAtTaOqc6AlnWeN8FSQQoGcIT48bnL89BtppSREF0NjgCaqtd32G33+EQx0G3hmKfAAWkGm355E9bnwaHLTd6NnoWnKSDjG8yybKgmc778HQuVpK1cnHQ2J83rZsK6x/xBJF4R9IvpQAAAABJRU5ErkJggg=='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Data Compression\"\n        title=\"Data Compression\"\n        src=\"/static/71ccccd89471c671316bf30da27bfd97/0a47e/datacompress.png\"\n        srcset=\"/static/71ccccd89471c671316bf30da27bfd97/8a4e8/datacompress.png 150w,\n/static/71ccccd89471c671316bf30da27bfd97/5a46d/datacompress.png 300w,\n/static/71ccccd89471c671316bf30da27bfd97/0a47e/datacompress.png 600w,\n/static/71ccccd89471c671316bf30da27bfd97/1cfc2/datacompress.png 900w,\n/static/71ccccd89471c671316bf30da27bfd97/60b3a/datacompress.png 1179w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h2 id=\"ideal-big-data-solution---main-technical-characteristics\" style=\"position:relative;\"><a href=\"#ideal-big-data-solution---main-technical-characteristics\" aria-label=\"ideal big data solution   main technical characteristics permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Ideal Big Data solution - main technical characteristics</h2>\n<ul>\n<li>Be scalable</li>\n<li>Fault Tolerant</li>\n<li>Ensure highly availability</li>\n<li>Ensure data is widely accessible, but secure</li>\n<li>Support analytics, data science, and content applications</li>\n<li>Support workflow automation</li>\n<li>Integrate with legacy applications</li>\n<li>Be self-healing (自愈)</li>\n</ul>\n<h3 id=\"functions\" style=\"position:relative;\"><a href=\"#functions\" aria-label=\"functions permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Functions</h3>\n<ul>\n<li>Data Collection</li>\n<li>Data Storage</li>\n<li>Data Exploration</li>\n<li>Data Governance</li>\n<li>Data Product</li>\n</ul>\n<p><span\n      class=\"gatsby-resp-image-wrapper\"\n      style=\"position: relative; display: block; margin-left: auto; margin-right: auto; max-width: 600px; \"\n    >\n      <a\n    class=\"gatsby-resp-image-link\"\n    href=\"/static/2627ac8ae993e00bbed971ce00a3cd5d/a2792/bigdata2.png\"\n    style=\"display: block\"\n    target=\"_blank\"\n    rel=\"noopener\"\n  >\n    <span\n    class=\"gatsby-resp-image-background-image\"\n    style=\"padding-bottom: 54%; position: relative; bottom: 0; left: 0; background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAALCAYAAAB/Ca1DAAAACXBIWXMAAA7DAAAOwwHHb6hkAAACP0lEQVQoz02S627TQBCF/f7PgoQE4i8gSoVEQ9uU0oY0TRx7ba9vu157fUkc53DWAYlIox3H6zPfzBnvdrfHriix2EfYlRqrWGIrM5hhQFUpCCGglEIoQuhK4ziOOJ5O6I9HZG2Plcyx3Id4jlOI2sK7Z7JVBm9fBK7jHG/WIT5v9gibFlGpkKgKdjhAsKjg8zCeUB+OGCgYUOCad59yjdsgmsW9VVkhMA3ev0a4CiRuRIpx6GD6AYukwCJVWEQpvrPYDeOZxX9n5UyYsOhdILAzFo9Jhk2u4C3DGK+FxseoxLdQ4sqP0fUUbFuUWqGsa+TuNAYyS2e6kWIdR+JAHkSCtapxz6KvHJm3FDF2SuMDCT/50UzYHwaYroM9HCCLHCKJEUoJTfGa5JZiSSohbYcn0jrCNaF22sB7jCT2TN5tBL6EKa4ZtrUomwbbKMLhfIZhXlPckfop3w89YpnMM7xjh2tdw3W6ppmenxfI6gZf4xIPeYWnoppbrkno2o3yDD4dDkmUaY2Ubbr52a6FXxn8jDO8VA2WNHdNWs+549al4sw0w7mnmho912OaTpBlgRPzhtQ9KbvjOL9zqxM3Fr/o7NZczk1BU+4oGLDl/i+Vm0/OfXN5RaKsKDByVVprUVBcmQoFDXKU+6rGjyDGii0/JDleLi5TkKacWLGlWMuds/NJYYrlXGpH3ZA+pUGa83TP89qwiNs937gzmzv1Jg59ms44ny8xTdMc7ncRKebckjgry8ud/+79++bf/38AoSE8J+fvvc4AAAAASUVORK5CYII='); background-size: cover; display: block;\"\n  ></span>\n  <img\n        class=\"gatsby-resp-image-image\"\n        alt=\"Storage &amp; formats of data\"\n        title=\"Storage &amp; formats of data\"\n        src=\"/static/2627ac8ae993e00bbed971ce00a3cd5d/0a47e/bigdata2.png\"\n        srcset=\"/static/2627ac8ae993e00bbed971ce00a3cd5d/8a4e8/bigdata2.png 150w,\n/static/2627ac8ae993e00bbed971ce00a3cd5d/5a46d/bigdata2.png 300w,\n/static/2627ac8ae993e00bbed971ce00a3cd5d/0a47e/bigdata2.png 600w,\n/static/2627ac8ae993e00bbed971ce00a3cd5d/1cfc2/bigdata2.png 900w,\n/static/2627ac8ae993e00bbed971ce00a3cd5d/c1b63/bigdata2.png 1200w,\n/static/2627ac8ae993e00bbed971ce00a3cd5d/a2792/bigdata2.png 1462w\"\n        sizes=\"(max-width: 600px) 100vw, 600px\"\n        style=\"width:100%;height:100%;margin:0;vertical-align:middle;position:absolute;top:0;left:0;\"\n        loading=\"lazy\"\n      />\n  </a>\n    </span></p>\n<h3 id=\"blue-print\" style=\"position:relative;\"><a href=\"#blue-print\" aria-label=\"blue print permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>Blue Print</h3>\n<ul>\n<li>ETL</li>\n<li>Streaming</li>\n<li>Batch</li>\n<li>Ingestion</li>\n<li>Exploratory Environment: development env using product data without impace product pipeline.</li>\n</ul>\n<h2 id=\"more\" style=\"position:relative;\"><a href=\"#more\" aria-label=\"more permalink\" class=\"anchor before\"><svg aria-hidden=\"true\" focusable=\"false\" height=\"16\" version=\"1.1\" viewBox=\"0 0 16 16\" width=\"16\"><path fill-rule=\"evenodd\" d=\"M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z\"></path></svg></a>More</h2>\n<ul>\n<li>Cloudera and Hortonworks are merged</li>\n</ul>","tableOfContents":"<ul>\n<li>\n<ul>\n<li><a href=\"#5-v\">5 V</a></li>\n<li><a href=\"#data-type\">data type</a></li>\n<li><a href=\"#speed-of-data\">Speed of data</a></li>\n</ul>\n</li>\n<li><a href=\"#storage--formats-of-data\">Storage &#x26; Formats of data</a></li>\n<li><a href=\"#data-compression\">Data compression</a></li>\n<li>\n<p><a href=\"#ideal-big-data-solution---main-technical-characteristics\">Ideal Big Data solution - main technical characteristics</a></p>\n<ul>\n<li><a href=\"#functions\">Functions</a></li>\n<li><a href=\"#blue-print\">Blue Print</a></li>\n</ul>\n</li>\n<li><a href=\"#more\">More</a></li>\n</ul>"},"previous":{"fields":{"id":"f9b1b128-a118-5569-b2ee-b6d5c347e964","title":"Github Radar","slug":"/bestitconsulting/radar/github-radar/"},"excerpt":"What is Big Data? According to wiki:  is a field that treats of ways to analyze, systematically extract information from, or otherwise deal…"},"next":{"fields":{"id":"36884170-cd04-548a-9a00-77e36a4b1380","title":"2 Hadoop","slug":"/bigdata/2-hadoop/"},"excerpt":"Hadoop The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of…"}},"pageContext":{"id":"17a02ec4-b391-54f1-8c60-922492f1a5eb","prevId":"f9b1b128-a118-5569-b2ee-b6d5c347e964","nextId":"36884170-cd04-548a-9a00-77e36a4b1380"}},"staticQueryHashes":["1576573137","63159454"]}